package com.ndood.spider.taobao.processor;

import java.net.URLDecoder;
import java.net.URLEncoder;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

import com.ndood.core.util.UrlStringUtil;
import com.ndood.spider.taobao.bean.TaobaoGoods;
import com.ndood.spider.taobao.dao.TaobaoGoodsDao;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Selectable;

/**
 * 方案一
 */
@Component
public class TaobaoPageProcessor implements PageProcessor {
	Logger logger = LoggerFactory.getLogger(getClass());
	
	@Autowired
	private TaobaoGoodsDao taobaoGoodsDao;

	@Override
	public Site getSite() {
		Site site = Site.me().setCharset("UTF-8").setCycleRetryTimes(3).setSleepTime(3 * 1000)
				.addHeader("Connection", "keep-alive").addHeader("Cache-Control", "max-age=0")
				.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:50.0) Gecko/20100101 Firefox/50.0");
		return site;
	}

	@Override
	public void process(Page page) {
		System.out.println(page.getJson());

		// Step1: 处理列表页
		if (page.getHtml().xpath("//*[@id='J_HesperCats']/ol/li/a").get() != null) {
			try {
				processListPage(page);
			} catch (Exception e) {
				e.printStackTrace();
			}
		}

		// Step2: 处理详情页
		if (page.getHtml().xpath("//*[@id='J_Title']/h3").get() != null) {
			try {
				processDetailsPage(page);
			} catch (Exception e) {
				e.printStackTrace();
			}
		}
	}

	private void processListPage(Page page) throws Exception {
		// Step1: 获取店铺ID
		String shopId = page.getHtml().xpath("//*[@id='bd']/div/div[@class='J_TRegion']/div[1]/@data-widgetid").get();
		
		// Step2: 加入所有分类列表 return
		if (page.getHtml().xpath("//*[@id='J_HesperCats']/ol/li[2]/a").get() == null) {
			List<Selectable> catNodes = page.getHtml()
					.xpath("//*[@id='shop" + shopId + "']/div/div[@class='skin-box-bd']/ul/li[@class='no-sub-cat']/h4/a")
					.nodes();
			for (Selectable catNode : catNodes) {
				String catUrl = catNode.xpath("//*/@href").get();
				page.addTargetRequest(catUrl);
			}
			return;
		}

		// Step3: 获取商品分类
		String cat = page.getHtml().xpath("//*[@id='J_HesperCats']/ol/li[2]/a/text()").get();
		String catPath = cat;
		String encodeCat = URLEncoder.encode(cat,"UTF-8");
		String encodeCatPath = encodeCat;
		
		// Step4: 加入下一页url
		String nextPageUrl = page.getHtml().xpath("//*[@id='J_ShopSearchResult']/div/div[@class='shop-hesper-bd']/div[@class='pagination']/a[@class='J_SearchAsync next']/@href").get();
		if(nextPageUrl!=null){
			page.addTargetRequest(nextPageUrl);
		}
		
		// Step5: 加入所有商品详情url
		List<Selectable> goodsNodes = page.getHtml().xpath("//*[@id='J_ShopSearchResult']/div/div[@class='shop-hesper-bd']/div[contains(@class,'line')]/dl[@class='item']").nodes();
		for (Selectable goodsNode : goodsNodes) {
			String goodsDetailUrl = goodsNode.xpath("//*/dt/a/@href").get();
			goodsDetailUrl = goodsDetailUrl + "&catPath="+encodeCatPath + "&cat=" + encodeCat;
			
			// id cat catPath url
			String id = UrlStringUtil.getParamsMap(goodsDetailUrl).get("id");
			TaobaoGoods goods = new TaobaoGoods();
			goods.setId(id);
			goods.setCat(cat);
			goods.setCatPath(catPath);
			goods.setUrl(goodsDetailUrl);
			try {
				TaobaoGoods temp = taobaoGoodsDao.get(id);
				if(temp==null){
					taobaoGoodsDao.insert(goods);
					// 只有新任务才需要爬详情页
					page.addTargetRequest(goodsDetailUrl);
				}else{
					// 如果该商品已包含该分类（防止重复执行添加）
					if(temp.getCat().contains(cat.trim())||temp.getCatPath().contains(catPath.trim())){
						continue;
					}
					// 如果商品已经存在则直接更新分类
					temp.setCat(temp.getCat()+","+cat);
					temp.setCatPath(temp.getCatPath()+","+catPath);
					taobaoGoodsDao.update(temp);
				}
			} catch (Exception e) {
				e.printStackTrace();
			}
		}
	}

	private void processDetailsPage(Page page) throws Exception {
		// 获取商品url
		String url = page.getUrl().get();
		if (!url.contains("id")) {
			return;
		}
		TaobaoGoods goods = new TaobaoGoods();
		goods.setUrl(url);
		
		// 店铺名称
		String shopName = page.getHtml().xpath("//*[@id='J_ShopInfo']/div[@class='tb-shop-info-wrap']/div[@class='tb-shop-info-hd']/div[@class='tb-shop-name']/dl/dd/strong/a/@title").get();
		goods.setShopName(shopName);

		// 获取商品ID
		Map<String, String> paramsMap = UrlStringUtil.getParamsMap(url);
		String id = paramsMap.get("id");
		goods.setId(id);
		
		// 获取商品分类
		String cat = paramsMap.get("cat");
		if(cat!=null){
			goods.setCat(URLDecoder.decode(cat, "UTF-8"));
		}
		String catPath = paramsMap.get("catpath");
		if(catPath!=null){
			goods.setCatPath(URLDecoder.decode(catPath, "UTF-8"));
		}
		
		// 获取商品名称
		String name = page.getHtml().xpath("//*[@id='J_Title']/h3/text()").get();
		goods.setName(name);
		
		// 获取商品价格
		String price = page.getHtml().xpath("//*[@id='J_PromoPriceNum']/text()").get();
		goods.setPrice(price);
		
		// 获取商品图片
		Map<String, String> imgUrls = new HashMap<String, String>();
		List<Selectable> imageNodes = page.getHtml().xpath("//*[@id='J_UlThumb']/li/div/a/img").nodes();
		for (int i = 0; i < imageNodes.size(); i++) {
			Selectable imageNode = imageNodes.get(i);
			String goodsImgUrl = imageNode.xpath("//*/@src").get();
			goodsImgUrl = goodsImgUrl.replaceAll("_([0-9]+)x([a-z0-9]+).jpg", "");
			imgUrls.put(name + i, goodsImgUrl);
		}
		goods.setImgUrls(imgUrls);
		
		// 获取销量和评论数
		String commentsCount = page.getHtml().xpath("//*[@id='J_RateCounter']/text()").get();
		String salesCount = page.getHtml().xpath("//*[@id='J_SellCounter']/text()").get();
		goods.setCommentsCount(commentsCount);
		goods.setSalesCount(salesCount);
		
		// 获取商品参数
		List<Selectable> goodsDescNodes = page.getHtml().xpath("//*[@id='attributes']/ul/li").nodes();
		for (int i=0;i<goodsDescNodes.size();i++) {
			Selectable goodsDescNode = goodsDescNodes.get(i);
			String line = goodsDescNode.xpath("//*/text()").get();
			String key = "";
			String value = "";
			if(line.split(":").length< 2){
				key = line.split("：")[0];
				value = line.split("：")[1];
			}else{
				key = line.split(":")[0];
				value = line.split(":")[1];
			}
			if(key.contains("品牌")){
				goods.setBrand(value);
			}else{
				goods.getParams().put(key, value);
			}
		}
		
		// 获取商品描述
		List<Selectable> descImgs = page.getHtml().xpath("//*[@id='J_DivItemDesc']/div/p/img").nodes();
		if(descImgs==null||descImgs.size()==0){
			descImgs = page.getHtml().xpath("//*[@id='J_DivItemDesc']/p/img").nodes();
		}
		if(descImgs==null||descImgs.size()==0){
			descImgs = page.getHtml().xpath("//*[@id='J_DivItemDesc']/div/div/img").nodes();
		}
		if(descImgs==null||descImgs.size()==0){
			descImgs = page.getHtml().xpath("//*[@id='J_DivItemDesc']/div/img").nodes();
		}
		if(descImgs==null||descImgs.size()==0){
			descImgs = page.getHtml().xpath("//*[@id='J_DivItemDesc']/div/p/span/img").nodes();
		}
		Map<String, String> descUrls = new HashMap<String, String>();
		for (int i = 0; i < descImgs.size(); i++) {
			Selectable descImg = descImgs.get(i);
			String descUrl = descImg.xpath("//*/@src").get();
			descUrl = descUrl.replaceAll("_([0-9]+)x([a-z0-9]+).jpg", "").replaceAll("https:", "");
			descUrls.put("商品描述" + i, descUrl);
		}
		goods.setDescUrls(descUrls);

		// 存入內存
		page.putField("goods" + id, goods);
	}

}